//
//  ScimPinyin.m
//  cocoa_py
//  $Id: ScimPinyin.mm 60 2010-08-07 22:00:18Z qiaolun $
//  Created by Qiao Lun on 6/21/10.
//  Copyright 2010 KK Group. All rights reserved.
//



#include <sys/stat.h>

#import "ScimPinyin.h"

#import "CocoaPinyinInputController.h"
#import "CocoaPinyinApplicationDelegate.h"

#define SCIM_PINYIN_MAX_USER_PHRASE_LENGTH   15
#define SCIM_PINYIN_MAX_SMART_MATCH_LEVEL    20
#define SCIM_PINYIN_MAX_PREEDIT_LENGTH       80

#define ENABLE_DEBUG 0


@implementation ScimPinyin

@synthesize encodingStr;
@synthesize user_dir;
@synthesize user_pinyin_table_path;
@synthesize user_phrase_lib_path;
@synthesize user_pinyin_phrase_lib_path;
@synthesize user_pinyin_phrase_index_path;

- (NSString *) getInputedString {
	return [NSString stringWithUTF8String:m_inputed_string.c_str()];
}

- (NSString *) getConvertedString {
	return [NSString stringWithUTF8String:utf8_wcstombs(m_converted_string).c_str()];
}

- (void)awakeFromNib {
	[self loadTable];
}

- (id) init {
	return [self initWithEncoding:@"GB2312"];
}

- (id) initWithEncoding:(NSString *)enc {

	self = [super init];
	if(self){
		
		m_dynamic_sensitivity = 16;
		m_smart_match_level = SCIM_PINYIN_MAX_SMART_MATCH_LEVEL;
		m_max_user_phrase_length = SCIM_PINYIN_MAX_USER_PHRASE_LENGTH;
		
		m_save_period = 30;
		
		
		
		std::string strDir;
		
		FSRef fsRef;
		if (FSFindFolder(kUserDomain, kApplicationSupportFolderType, kCreateFolder, &fsRef) == noErr) {
			CFURLRef urlRef = CFURLCreateFromFSRef(NULL, &fsRef);
			if (urlRef != NULL) {
				UInt8 buffer[PATH_MAX + 1]; 
				if (CFURLGetFileSystemRepresentation(urlRef, true, buffer, sizeof(buffer))) {
					strDir.assign((char*) buffer);
				}   
				CFRelease(urlRef) ;
			}   
		}   
		
		
		[self setUser_dir:[[NSString stringWithUTF8String:strDir.c_str()] stringByAppendingPathComponent:@"/QLCocoaPinyin"]];
		NSLog(@"home dir %@", [self user_dir]);
		
		[self setUser_phrase_lib_path: [[self user_dir] stringByAppendingPathComponent:@"phrase_lib"]];
		[self setUser_pinyin_phrase_lib_path: [[self user_dir] stringByAppendingPathComponent:@"pinyin_phrase_lib"]];
		[self setUser_pinyin_phrase_index_path: [[self user_dir] stringByAppendingPathComponent:@"pinyin_phrase_index"]];
		NSLog(@"user phrase lib path %@", [self user_phrase_lib_path]);
		
		m_pinyin_global = new PinyinGlobal ();
		m_pinyin_global->toggle_tone (false);
		m_pinyin_global->toggle_incomplete (true);
		m_pinyin_global->toggle_dynamic_adjust (true);
		m_pinyin_global->update_custom_settings ();
		
		m_pinyin_parser = new PinyinDefaultParser ();
		
		[self setEncodingStr:enc];
		
		
		
	}
	return self;
}


- (BOOL)loadTable {

	
	size_t size;
	
	
	// get pinyin table
	NSString *f_sys_pinyin_table = [[NSBundle mainBundle] pathForResource:@"pinyin_table" ofType:nil];
	[self setUser_pinyin_table_path: [[self user_dir] stringByAppendingPathComponent:@"pinyin_table"]];
	if(f_sys_pinyin_table){
		if(!m_pinyin_global->load_pinyin_table([f_sys_pinyin_table UTF8String], [[self user_pinyin_table_path] UTF8String])){
			NSLog(@"cannot load pinyin table sys");
			// return FALSE;
		}
	}
	
	m_pinyin_table = m_pinyin_global->get_pinyin_table();
	size = m_pinyin_table->size();
	NSLog(@"pinyin table size: %d", size);
	
	// get sys phrase lib
	NSString *f_sys_phrase_lib = [[NSBundle mainBundle] pathForResource:@"phrase_lib" ofType:nil];
	NSString *f_sys_pinyin_phrase_lib = [[NSBundle mainBundle] pathForResource:@"pinyin_phrase_lib" ofType:nil];
	NSString *f_sys_pinyin_phrase_index = [[NSBundle mainBundle] pathForResource:@"pinyin_phrase_index" ofType:nil];
	
	if(f_sys_phrase_lib && f_sys_pinyin_phrase_lib && f_sys_pinyin_phrase_index){
		if(!m_pinyin_global->load_sys_phrase_lib([f_sys_phrase_lib UTF8String],
												 [f_sys_pinyin_phrase_lib UTF8String],
												 [f_sys_pinyin_phrase_index UTF8String])){
			NSLog(@"cannot load sys phrase lib");
			// return FALSE;
		}
	}
	
	m_sys_phrase_lib = m_pinyin_global->get_sys_phrase_lib();
	size = m_sys_phrase_lib->number_of_phrases();
	NSLog(@"sys phrase size: %d", size);
	
	
	// get user phrase lib
	if([self user_phrase_lib_path] 
	   && [self user_pinyin_phrase_lib_path] 
	   && [self user_pinyin_phrase_index_path]){
		if(!m_pinyin_global->load_user_phrase_lib([[self user_phrase_lib_path] UTF8String],
												 [[self user_pinyin_phrase_lib_path] UTF8String],
												 [[self user_pinyin_phrase_index_path] UTF8String])){
			NSLog(@"cannot load user phrase lib");
			// return FALSE;
		}
	}
	
	m_user_phrase_lib = m_pinyin_global->get_user_phrase_lib();
	size = m_user_phrase_lib->number_of_phrases();
	NSLog(@"user phrase size: %d", size);

	m_pinyin_global->get_user_phrase_lib ()->set_burst_stack_size (128);
	
	m_last_time = time (0);

	
	[self reset];
	
	
	return TRUE;
}

-(void) refresh {
    if (m_save_period == 0)
        return;
	
    time_t cur_time = time (0);
	
    if (cur_time < m_last_time || cur_time - m_last_time > m_save_period) {
        m_last_time = cur_time;
        [self saveUserLibrary];
    }
}


- (void) saveUserLibrary {

	NSLog(@"try save user libraryzheyang at %@", [self user_dir]);

	
	// First make the user data directory.
	BOOL isDir;
	if ([[NSFileManager defaultManager] fileExistsAtPath:[self user_dir] isDirectory:&isDir] && isDir) {
		// dir exists
	} else {
		// create dir
		NSLog(@"create user data directory. %@", user_dir);
		[[NSFileManager defaultManager] createDirectoryAtPath:[self user_dir] withIntermediateDirectories:YES attributes:nil error:NULL];
	}

	
	/*
	if (access ([user_dir UTF8String], R_OK | W_OK) != 0) {
		mkdir ([user_dir UTF8String], S_IRUSR | S_IWUSR | S_IXUSR);
		NSLog(@"create user data directory.");
		if (access ([user_dir UTF8String], R_OK | W_OK) != 0)
			NSLog(@"still cannot access user data directory.");
			return;
		 
	}
	*/
	

	
	PinyinPhraseLib *lib = m_pinyin_global->get_user_phrase_lib ();
	
    if (lib) {
        lib->optimize_phrase_relation_map ();
        lib->optimize_phrase_frequencies ();
    }    
	

	
    m_pinyin_global->save_pinyin_table ([user_pinyin_table_path UTF8String], false);
    m_pinyin_global->save_user_phrase_lib ([user_phrase_lib_path UTF8String],
                                          [user_pinyin_phrase_lib_path UTF8String],
                                          [user_pinyin_phrase_index_path UTF8String],
                                          false);
	
	
}

- (void) reset{
	
	if([encodingStr isEqualToString:@"GB2312"] || [encodingStr isEqualToString:@"BIG5"]) {
		// OK
	} else {
		[self setEncodingStr:@"UTF-8"];
	}
	
	m_iconv.set_encoding([encodingStr UTF8String]);
    m_chinese_iconv.set_encoding ([encodingStr UTF8String]);
	
	m_inputed_string = String ();
    
    m_converted_string = WideString ();
	
    std::vector <PinyinParsedKey> ().swap (m_parsed_keys);
    std::vector <PhraseVector> ().swap (m_phrases_cache);
    std::vector <CharVector> ().swap (m_chars_cache);
	
	[self clear_selected: 0];
	
    m_keys_caret = 0;
    m_lookup_caret = 0;
	
	m_lookup_table.set_page_size(9);
	m_lookup_table.clear ();
	
	PYCandidates *candidatesWindow = [[NSApp delegate] candidates];
	[candidatesWindow hide:nil];
}

- (BOOL) inputString:(NSString *)str client:(id)sender {
	
	
	// NSLog(@"converted_string %d, parsed_keys %d", m_converted_string.length(), m_parsed_keys.size () );
	// NSLog(@"input %@", str);

	
	// check input key
	char key = [str characterAtIndex:0];
	if ((key >= 'a' && key <= 'z') || ( m_inputed_string.length () && key == '\'') ) {
		// input validation ok
	
	} else if(m_inputed_string.length () && (key >= '1' && key <= '9') ) {
		// select key
		int index = key - '1';
		[self lookup_select:index client:sender];
		return TRUE;
	} else if(m_inputed_string.length () && (key == ',' || key == '.') ) {
		// page up/down
		if(key == '.'){
			[self lookup_page_down:sender];
		} else if(key == ','){
			[self lookup_page_up:sender];
		}
		return TRUE;
	} else if(key == ' '){
		return [self space_hit:sender];
	} else {
		return FALSE;
	}
	
	
	
	std::vector <PinyinParsedKey> old_parsed_keys = m_parsed_keys;
    String old_inputed_string = m_inputed_string;
    PinyinKeyExactEqualTo equal;
	
	int inputed_caret = [self calc_inputed_caret];
	
	if ((m_parsed_keys.size () != 0 &&
		 m_inputed_string.length () - m_parsed_keys.back ().get_end_pos () > SCIM_PINYIN_KEY_MAXLEN) ||
		(m_parsed_keys.size () == 0 &&
		 m_inputed_string.length () > SCIM_PINYIN_KEY_MAXLEN))
		return TRUE;
	
	if (inputed_caret == 0 && ((key >= '1' && key <= '5') || key == '\'' || key == ';'))
		return [self post_process:key client:sender ];
	
	String::iterator i = m_inputed_string.begin () + inputed_caret;
	
	if (key != '\'') {
		m_inputed_string.insert (i, key);
	} else {
		// Don't insert more than one split chars in one place.
		if ((i != m_inputed_string.begin () && *(i-1) == '\'') ||
			(i != m_inputed_string.end () && *i == '\''))
			return TRUE;
		m_inputed_string.insert (i, key);
	}
	
	[self calc_parsed_keys];
	
	if (m_parsed_keys.size () > SCIM_PINYIN_MAX_PREEDIT_LENGTH) {
		m_inputed_string = old_inputed_string;
		m_parsed_keys = old_parsed_keys;
		return TRUE;
	}
	
	unsigned int len;
	for (len = 0; len < m_parsed_keys.size () && len < old_parsed_keys.size (); ++ len) {
		if (!equal (m_parsed_keys [len], old_parsed_keys [len]))
			break;
	}
	
	if (m_converted_string.length () > len)
		m_converted_string.erase (m_converted_string.begin () + len, m_converted_string.end ());
	
	m_keys_caret = [self inputed_caret_to_key_index:(inputed_caret + 1)];
	
	if (m_keys_caret <= (int) m_converted_string.length ())
		m_lookup_caret = m_keys_caret;
	
	else if (m_lookup_caret > (int) m_converted_string.length ())
		m_lookup_caret = m_converted_string.length ();

	// converted_string + parsed_keys[lookup_caret]
	
	[self updateMarkedText:sender];
	[self refresh_lookup_table:len client:sender];
	
	return TRUE;
}


- (void) updateMarkedText:(id)sender {
	
	/*
	PinyinParsedKeyVector::const_iterator begin   = m_parsed_keys.begin () + m_lookup_caret;
	PinyinParsedKeyVector::const_iterator end     = m_parsed_keys.end ();
	
	for(PinyinParsedKeyVector::const_iterator i = begin ; i != end; ++i) {
		std::cerr << "parsed key: " << (*i).get_key_string() << std::endl ;
	}
	*/
	
	[self calc_preedit_string];
	// NSLog(@"preedit string %@", [NSString stringWithUTF8String:utf8_wcstombs(m_preedit_string).c_str()]);
	
	
	NSString *buffer = [NSString stringWithUTF8String:utf8_wcstombs(m_preedit_string).c_str()];
	[sender setMarkedText:buffer selectionRange:NSMakeRange(0, [buffer length]) replacementRange:NSMakeRange(NSNotFound, NSNotFound)];

}


- (BOOL) post_process:(char) key client:(id)sender
{
    if (m_inputed_string.length () > 0) {
        if (m_converted_string.length () == m_parsed_keys.size () && ![self has_unparsed_chars]) {
            [self commit_converted:sender ];
            [self refresh_lookup_table:0 client:sender];
        } else {
            return TRUE;
        }
    }
	
	/*
    if ((ispunct (key) && m_full_width_punctuation [m_forward?1:0]) ||
        ((isalnum (key) || key == 0x20) && m_full_width_letter [m_forward?1:0])) {
        commit_string (convert_to_full_width (key));
        return true;
    }
	*/
	
    return FALSE;
}

- (void) commit_converted:(id)sender
{
    if (m_converted_string.length ()) {
		
        [self commit_string:m_converted_string client:sender];
		
		if (m_pinyin_global && m_pinyin_global->use_dynamic_adjust ()) {
            [self dynamic_adjust_selected];
            [self add_new_phrase:m_converted_string keys:m_parsed_keys ref:false];
            [self clear_selected: 0];
            [self refresh];
        }
		
        if (m_converted_string.length () <= m_parsed_keys.size ()) {
            m_keys_caret -= m_converted_string.length ();
            m_inputed_string.erase (0, m_parsed_keys [m_converted_string.length () - 1].get_end_pos ());
        } else {
            m_keys_caret -= m_parsed_keys.size ();
            m_inputed_string.erase (0, m_parsed_keys.back ().get_end_pos ());
        }
		
        if (m_keys_caret < 0) m_keys_caret = 0;
		
        m_converted_string = WideString ();
        m_lookup_caret = 0;
		
        [self calc_parsed_keys];
    }
}


- (bool) has_unparsed_chars 
{
    if (m_inputed_string.length () == 0) 
        return false;
    if (m_parsed_keys.size () == 0)
        return true;
    if (m_parsed_keys.back ().get_end_pos () < (int)m_inputed_string.length ())
        return true;
    return false;
}

- (int) inputed_caret_to_key_index: (int) caret
{
    if (m_parsed_keys.size () == 0) {
        if (caret > 0) return 1;
        else return 0;
    }
	
    for (int i=0; i<(int)m_parsed_keys.size (); i++) {
        if (caret >= m_parsed_keys [i].get_pos () && caret < m_parsed_keys [i].get_end_pos ())
            return i;
    }
	
    if (caret == m_parsed_keys.back ().get_end_pos ())
        return m_parsed_keys.size ();
	
    return m_parsed_keys.size () + 1;
}

- (int)calc_inputed_caret
{
    int caret;
    if (m_keys_caret <= 0)
        caret = 0;
    else if (m_keys_caret < (int) m_parsed_keys.size ()) {
        caret = m_parsed_keys [m_keys_caret].get_pos ();
    } else if (m_keys_caret == (int) m_parsed_keys.size ()) {
        caret = m_parsed_keys [m_keys_caret-1].get_end_pos ();
        if (caret < (int)m_inputed_string.length () && m_inputed_string [caret] == '\'')
            caret ++;
    } else {
        caret = m_inputed_string.length ();
    }
	
    return caret;
}

- (void) calc_parsed_keys 
{
    m_pinyin_parser->parse (m_pinyin_global->get_pinyin_validator (),
                                       m_parsed_keys,
                                       m_inputed_string.c_str ());
}

- (NSArray *) get_candidates {
	if (m_lookup_table.get_current_page_size()) {
		uint32 count = m_lookup_table.get_current_page_size();
		NSMutableArray *arr = [NSMutableArray arrayWithCapacity:count];
		
		for(int i=0; i<count; i++){
			WideString candy = m_lookup_table.get_candidate_in_current_page(i);
			
			// std::cerr << i << " " << utf8_wcstombs(candy) << std::endl ;
			
			// NSLog(@"candidate %d %s", i, m_lookup_table.get_candidate(i).c_str() );
			
			NSString *wd = [NSString stringWithUTF8String:(utf8_wcstombs(candy).c_str())];
			
			[arr insertObject:wd atIndex:i];
		}
		
		return [NSArray arrayWithArray:arr];
	} else {
		return nil;
	}
}

- (void) refresh_lookup_table: (int) invalid_pos client:(id)sender
{
    [self calc_lookup_table:invalid_pos];
	
	// extern IMKCandidates* candidates;
	
    [self update_lookup_table:sender];
}


- (void) calc_lookup_table: (int) invalid_pos
{


	m_lookup_table.clear();

    if (m_parsed_keys.size () == 0) return;
	
    WideString str; 
    bool       longer_phrase = false;
	
    PinyinParsedKeyVector::const_iterator begin   = m_parsed_keys.begin () + m_lookup_caret;
    PinyinParsedKeyVector::const_iterator end     = m_parsed_keys.end ();
    PinyinParsedKeyVector::const_iterator invalid;
	
    if (begin >= end) return;
	
    if (invalid_pos >= 0)
        invalid = m_parsed_keys.begin () + invalid_pos;
    else
        invalid = end;

    
	scim_pinyin_update_matches_cache (
									  m_chars_cache,
									  m_phrases_cache,
									  m_parsed_keys.begin (),
									  m_parsed_keys.end (),
									  invalid,
									  m_pinyin_table,
									  m_user_phrase_lib,
									  m_sys_phrase_lib,
									  &m_iconv,
									  &m_chinese_iconv,
									  false,
									  longer_phrase);
	
    // If auto_combine_phrase is enable, then use smart match func
    // to create the content of lookup table
    if (true) {
		
        WideString acp;
        WideString first;
		
        PhraseVector mphrases;
		
        acp = scim_pinyin_smart_match (
									   mphrases,
									   m_chars_cache.begin () + m_lookup_caret,
									   m_phrases_cache.begin () + m_lookup_caret,
									   begin,
									   end,
									   m_pinyin_table,
									   m_user_phrase_lib,
									   m_sys_phrase_lib,
									   SCIM_PINYIN_MAX_SMART_MATCH_LEVEL,
									   &m_iconv,
									   &m_chinese_iconv);
		
        if (m_phrases_cache [m_lookup_caret].size ()) {
            first = m_phrases_cache [m_lookup_caret] [0].get_content ();
        } else if (m_chars_cache [m_lookup_caret].size ()) {
            first += m_chars_cache [m_lookup_caret] [0];
        }
		
        if (true && first != acp && acp.length ()) {
            m_lookup_table.append_entry(acp);
        }
		
    }
	
    if (!m_phrases_cache [m_lookup_caret].size () || !m_chars_cache [m_lookup_caret].size ()) {

		scim_pinyin_search_matches (
									m_chars_cache [m_lookup_caret],
									m_phrases_cache [m_lookup_caret],
									begin,
									end,
									m_pinyin_table,
									m_user_phrase_lib,
									m_sys_phrase_lib,
									&m_iconv,
									&m_chinese_iconv,
									true,
									longer_phrase);

	}

    if (m_phrases_cache [m_lookup_caret].size ()) {
        for (PhraseVector::iterator i = m_phrases_cache [m_lookup_caret].begin ();
			 i != m_phrases_cache [m_lookup_caret].end (); ++ i)
            m_lookup_table.append_entry (*i);
    }

    if (m_chars_cache [m_lookup_caret].size ()) {
        for (CharVector::iterator i = m_chars_cache [m_lookup_caret].begin ();
			 i != m_chars_cache [m_lookup_caret].end (); ++ i)
            m_lookup_table.append_entry (*i);
    }
}


- (BOOL)lookup_page_up:(id)sender {
    if (m_inputed_string.length () && m_lookup_table.number_of_candidates ()) {
        m_lookup_table.page_up ();
        m_lookup_table.set_page_size (9);
        [self update_lookup_table:sender];
        return TRUE;
    }    
    return FALSE;
}

- (BOOL)lookup_page_down:(id)sender {
    if (m_inputed_string.length () && m_lookup_table.number_of_candidates ()) {
        m_lookup_table.page_down ();
        m_lookup_table.set_page_size (9);
        [self update_lookup_table:sender];
        return TRUE;
    }    
    return FALSE;
}

- (void) update_lookup_table:(id)sender {
	
	if (m_lookup_table.number_of_candidates()) {
		// show lookup table
		// NSLog(@"got candidates");
		
		/*
		uint32 count = m_lookup_table.number_of_candidates();
		for(int i=0; i<count; i++){
			WideString candy = m_lookup_table.get_candidate(i);
			
			std::cerr << i << " " << utf8_wcstombs(candy) << std::endl ;
			
			NSString *wd = [NSString stringWithUTF8String:(utf8_wcstombs(candy).c_str())];
			
			NSLog(@"candidate %d %@", i, wd );
			
		}
		*/
		/*
		 if ( candidates ) {
		 [candidates updateCandidates];
		 [candidates show:kIMKLocateCandidatesBelowHint];
		 }
		 */
		
		NSArray *candidatesData = [self get_candidates];
		PYCandidates *candidatesWindow = [[NSApp delegate] candidates];
		[candidatesWindow showCandidates:candidatesData client:sender];
		
    } else {
		// hide lookup table
		// NSLog(@"no candidate");
		/*
		 if ( candidates ) {
		 [candidates hide];
		 }
		 */
		
		PYCandidates *candidatesWindow = [[NSApp delegate] candidates];
		[candidatesWindow hide:sender];
	}
}

- (BOOL) lookup_select: (int) index client:(id)sender
{
    if (m_inputed_string.length ()) {
        if (m_lookup_table.number_of_candidates () == 0)
            return TRUE;
		
		index += m_lookup_table.get_current_page_start ();
		
        [self lookup_to_converted:index];
		
        int invalid_pos = -1;
		
        if (m_converted_string.length () >= m_parsed_keys.size () &&
            m_lookup_caret == (int) m_converted_string.length ()) {
            [self commit_converted:sender];
            invalid_pos = 0;
        }
		
		[self updateMarkedText:sender];
        [self refresh_lookup_table:invalid_pos client:sender];
		
		// NSLog(@"lookup caret %d", m_lookup_caret);
		// NSLog(@"converted string %@", [NSString stringWithUTF8String:utf8_wcstombs(m_converted_string).c_str()]);
		
        return TRUE;
    }
	
    return FALSE;
}


- (void) lookup_to_converted: (int) index
{
    if (index < 0 || index >= (int) m_lookup_table.number_of_candidates ())
        return;
	
    WideString str = m_lookup_table.get_candidate (index);
	
    if (m_lookup_caret < (int) m_converted_string.length ()) {
        m_converted_string.erase (m_lookup_caret, min (str.length (), m_converted_string.length () - m_lookup_caret));
    }

    m_converted_string.insert (m_lookup_caret, str);
	
	if (m_pinyin_global && m_pinyin_global->use_dynamic_adjust ()) {
        if (m_lookup_table.is_string (index)) {
            [self store_selected_string:m_lookup_caret sstring: str stotal: m_converted_string];
        } else if (m_lookup_table.is_phrase (index)) {
            [self store_selected_phrase: m_lookup_caret sphrase: m_lookup_table.get_phrase (index) stotal: m_converted_string];
        } else {   
            Phrase phrase;
            if (m_user_phrase_lib && m_user_phrase_lib->valid ()) {
                phrase = m_user_phrase_lib->find (str);
			}
            if (!phrase.valid () && m_sys_phrase_lib && m_sys_phrase_lib->valid ()) {
                phrase = m_sys_phrase_lib->find (str);
			}
            if (phrase.valid ()) {
                [self store_selected_phrase:m_lookup_caret sphrase:phrase stotal:m_converted_string];
			}
        }            
    } 
	
    m_lookup_caret += str.length ();
	
    if (m_keys_caret < m_lookup_caret)
        m_keys_caret = m_lookup_caret;
}


- (BOOL)erase:(BOOL) backspace client:(id)sender
{
    if (m_converted_string.length() && m_lookup_caret > 0){
		m_converted_string.erase (m_lookup_caret - 1, 1);
		m_lookup_caret--;
		
		[self updateMarkedText:sender];		
		[self refresh_lookup_table:-1 client:sender];
		
		return TRUE;
	}
	
	if (m_inputed_string.length ()) {
        std::vector <PinyinParsedKey> old_parsed_keys = m_parsed_keys;
		
        int inputed_caret = [self calc_inputed_caret];
		
        if (!backspace && inputed_caret < (int) m_inputed_string.length ())
            inputed_caret ++;
		
        if (inputed_caret > 0) {
            m_inputed_string.erase (inputed_caret - 1, 1);

            [self calc_parsed_keys];
			
            m_keys_caret = [self inputed_caret_to_key_index: (inputed_caret - 1)];
			
            unsigned int len;
            PinyinKeyExactEqualTo equal;
			
            for (len = 0; len < m_parsed_keys.size () && len < old_parsed_keys.size (); len ++) {
                if (!equal (m_parsed_keys [len], old_parsed_keys [len]))
                    break;
            }
			
            if (m_converted_string.length () > len)
                m_converted_string.erase (m_converted_string.begin () + len, m_converted_string.end ());
			
            if (m_keys_caret <= (int) m_converted_string.length () &&
                m_lookup_caret > m_keys_caret)
                m_lookup_caret = m_keys_caret;
            else if (m_lookup_caret > (int) m_converted_string.length ())
                m_lookup_caret = m_converted_string.length ();
			
			[self updateMarkedText:sender];		
            [self refresh_lookup_table:len client:sender];
        }
		

		
        return TRUE;
    }
    return FALSE;
}


- (BOOL) space_hit:(id)sender
{
    if (m_inputed_string.length ()) {
        if (m_converted_string.length () == 0 && m_lookup_table.number_of_candidates () == 0)
            return TRUE;
		
		if (m_lookup_table.number_of_candidates () &&
            (m_converted_string.length () <= m_parsed_keys.size () ||
             m_keys_caret == m_lookup_caret))
            [self lookup_to_converted:m_lookup_table.get_cursor_pos()];
		
		// std::cerr << "space_hit, converted string: " << utf8_wcstombs(m_converted_string) << std::endl;
		
        int invalid_pos = -1;
		
        if (m_converted_string.length () >= m_parsed_keys.size ()) {
            [self commit_converted:sender];
            invalid_pos = 0;
        }
		
		[self updateMarkedText:sender];
        [self refresh_lookup_table:invalid_pos client:sender];
		
        return TRUE;
    }
    return FALSE;
}

- (BOOL) enter_hit:(id)sender
{
    if (m_inputed_string.length ()) {
        WideString str = utf8_mbstowcs (m_inputed_string);
        [self reset];
        [self commit_string:str client:sender];
        return TRUE;
    }
    return FALSE;
}


- (void) calc_preedit_string
{
    m_preedit_string = WideString ();
	
    if (m_inputed_string.length () == 0) {
        return;
    }
	
    WideString unparsed_string;
    unsigned int i;
	
    m_preedit_string = m_converted_string;
	
    for (i = m_converted_string.length (); i<m_parsed_keys.size (); i++) {
        int endpos = m_parsed_keys [i].get_end_pos ();
        for (int j = m_parsed_keys [i].get_pos (); j<endpos; j++) {
            m_preedit_string += ((ucs4_t) m_inputed_string [j]);
        }
        m_preedit_string += (ucs4_t) 0x0020;
    }
	
    if (m_parsed_keys.size () == 0) {
        unparsed_string = utf8_mbstowcs (m_inputed_string);
    } else {
        int parsed_len = m_parsed_keys.back ().get_end_pos ();
        for (i=parsed_len; i<m_inputed_string.length (); i++) {
            unparsed_string += (ucs4_t) (m_inputed_string [i]);
        }
    }
	
    if (unparsed_string.length ())
        m_preedit_string += unparsed_string;
}



- (void) commit_string:(WideString)wstr client:(id)sender {

	NSString *text = [NSString stringWithUTF8String:utf8_wcstombs(wstr).c_str()];
	
	// NSLog(@"commit %@", text);
	[sender insertText:text replacementRange:NSMakeRange(NSNotFound, NSNotFound)];
}


- (void)store_selected_phrase: (int) caret sphrase:( const Phrase &)phrase stotal:(const WideString &)total {
    if (!phrase.length ()) return;
	
    unsigned int i;
	
    std::vector < std::pair<int, WideString> > tmp_strings;
    std::vector < std::pair<int, Phrase> > tmp_phrases;
	
    for (i = 0; i < m_selected_strings.size (); ++ i) {
        int old_start = m_selected_strings [i].first;
        int old_end = m_selected_strings [i].first + m_selected_strings [i].second.length ();
        int new_end = caret + phrase.length ();
		
        if (old_end <= caret || new_end <= old_start)
            tmp_strings.push_back (m_selected_strings [i]);
        else if (old_start <= caret && old_end >= new_end)
            tmp_strings.push_back (std::make_pair (old_start, total.substr (old_start, old_end - old_start)));
        else if (caret <= old_start && old_end > new_end)
            tmp_strings.push_back (std::make_pair (new_end, total.substr (new_end, old_end - new_end)));
        else if (old_start < caret && old_end <= new_end)
            tmp_strings.push_back (std::make_pair (old_start, total.substr (old_start, caret - old_start)));
    }
	
    for (i = 0; i < m_selected_phrases.size (); ++ i) {
        if ((m_selected_phrases [i].first + m_selected_phrases [i].second.length ()) <= caret ||
            (caret + phrase.length ()) <= m_selected_phrases [i].first)
            tmp_phrases.push_back (m_selected_phrases [i]);
    }
	
    tmp_phrases.push_back (std::make_pair (caret, phrase));
	
    m_selected_strings.swap (tmp_strings);
    m_selected_phrases.swap (tmp_phrases);
}

- (void) store_selected_string: (int) caret sstring:(const WideString &)str stotal: (const WideString &)total {
    unsigned int i;
	
    std::vector < std::pair<int, WideString> > tmp_strings;
    std::vector < std::pair<int, Phrase> > tmp_phrases;
	
    for (i = 0; i < m_selected_strings.size (); ++ i) {
        int old_start = m_selected_strings [i].first;
        int old_end = m_selected_strings [i].first + m_selected_strings [i].second.length ();
        int new_end = caret + str.length ();
		
        if (old_end <= caret || new_end <= old_start)
            tmp_strings.push_back (m_selected_strings [i]);
        else if (old_start <= caret && old_end >= new_end)
            tmp_strings.push_back (std::make_pair (old_start, total.substr (old_start, old_end - old_start)));
        else if (caret <= old_start && old_end > new_end)
            tmp_strings.push_back (std::make_pair (new_end, total.substr (new_end, old_end - new_end)));
        else if (old_start < caret && old_end <= new_end)
            tmp_strings.push_back (std::make_pair (old_start, total.substr (old_start, caret - old_start)));
    }
	
    for (i = 0; i < m_selected_phrases.size (); ++ i) {
        if ((m_selected_phrases [i].first + m_selected_phrases [i].second.length ()) <= caret ||
            (caret + str.length ()) <= m_selected_phrases [i].first)
            tmp_phrases.push_back (m_selected_phrases [i]);
    }
	
    tmp_strings.push_back (std::make_pair (caret, str));
	
    m_selected_strings.swap (tmp_strings);
    m_selected_phrases.swap (tmp_phrases);
}

- (void) dynamic_adjust_selected {
	
    if (m_user_phrase_lib && m_user_phrase_lib->valid ()) {
        PinyinParsedKeyVector keys;
        Phrase newph;
		
        std::vector <std::pair <int, Phrase> > added_phrases;
		
        unsigned int i;
		
        for (i = 0; i < m_selected_phrases.size (); ++ i) {
            if (m_selected_phrases [i].first + m_selected_phrases [i].second.length () <=
                m_parsed_keys.size ()) {
                keys = PinyinParsedKeyVector (
											  m_parsed_keys.begin () + m_selected_phrases [i].first,
											  m_parsed_keys.begin () + m_selected_phrases [i].first +
											  m_selected_phrases [i].second.length ());
                newph = [self add_new_phrase:m_selected_phrases [i].second.get_content () keys:keys ref:true];
				
                if (newph.valid ()) {
                    added_phrases.push_back (std::make_pair (m_selected_phrases [i].first, newph));
                }
			}
		}
		
		for (i = 0; i < m_selected_strings.size (); ++ i) {
			if (m_selected_strings [i].first + m_selected_strings [i].second.length () <=
				m_parsed_keys.size ()) {
				keys = PinyinParsedKeyVector (
											  m_parsed_keys.begin () + m_selected_strings [i].first,
											  m_parsed_keys.begin () + m_selected_strings [i].first +
											  m_selected_strings [i].second.length ());
				
				if (keys.size ()) {
					newph = [self add_new_phrase: m_selected_strings [i].second keys:keys ref:true];
					if (newph.valid ()) {
						added_phrases.push_back (std::make_pair (m_selected_strings [i].first, newph));
					}
				}
			}
		}
		
		// Try to find out and add new inputed phrase
		WideString phrase;
		unsigned int last_start = 0;
		
		//sort by caret
		std::sort (added_phrases.begin (), added_phrases.end ());
		
#if ENABLE_DEBUG
		std::cerr << "Added new phrases :\n";
		for (i=0; i<added_phrases.size (); ++ i) {
			std::cerr << added_phrases [i].first << " = ";
			std::cerr << utf8_wcstombs (added_phrases [i].second.get_content ()) << "\n";
		}
#endif
		
		unsigned int num = added_phrases.size ();
		
		// Combine several single chars to phrases.
		for (i = 1; i < num; ++ i) {
			if (added_phrases [i-1].first + 1 != added_phrases [i].first ||
				added_phrases [i].second.length () != 1) {
				
				if (added_phrases [i-1].second.length () == 1) {
					phrase += added_phrases [i-1].second.get_content ();
					if (phrase.length () > 1) {
						keys = PinyinParsedKeyVector (
													  m_parsed_keys.begin () + added_phrases [last_start].first,
													  m_parsed_keys.begin () + added_phrases [last_start].first +
													  phrase.length ());
						
						newph = [self add_new_phrase:phrase keys:keys ref:true];
						if (newph.valid ()) {
							added_phrases.push_back (
													 std::make_pair (added_phrases [last_start].first, newph));
						}
					}
				}
				phrase.clear ();
				last_start = i;
			} else {
				phrase += added_phrases [i-1].second.get_content ();
			}
		}
		
		if (phrase.length () >= 1 && added_phrases [num-1].second.length () == 1) {
			phrase += added_phrases [num-1].second.get_content ();
			keys = PinyinParsedKeyVector (
										  m_parsed_keys.begin () + added_phrases [last_start].first,
										  m_parsed_keys.begin () + added_phrases [last_start].first +
										  phrase.length ());
			newph = [self add_new_phrase:phrase keys:keys ref:true];
			if (newph.valid ()) {
				added_phrases.push_back (
										 std::make_pair (added_phrases [last_start].first, newph));
			}
		}
		
		// refresh phrases relation data
		for (i = 1; i < added_phrases.size (); ++ i) {
			if (added_phrases [i-1].first + added_phrases [i-1].second.length () == added_phrases [i].first &&
				(added_phrases [i-1].second.length () + added_phrases [i].second.length () > 2))
				m_user_phrase_lib->refresh_phrase_relation (
															added_phrases [i-1].second,
															added_phrases [i].second,
															16);
		}
	}
}

				
- (Phrase) add_new_phrase: (const WideString &)phrase keys:(const PinyinParsedKeyVector &)pkeys ref:(bool) refresh
{
	Phrase newph;
	
#if ENABLE_DEBUG
	std::cerr << "Add New Phrase: " << utf8_wcstombs (phrase) << " (";
	for (unsigned int i=0; i<pkeys.size (); ++i) {
		std::cerr << pkeys [i] << " ";
	}
	std::cerr << ")\n";
#endif
	
	if (m_user_phrase_lib && m_user_phrase_lib->valid () && phrase.length () >= 1) {
		/*
		 * First check if the phrase is already in user phrase library,
		 * if not, then add it, otherwise refresh it.
		 */
		newph = m_user_phrase_lib->find (phrase);
		if (!newph.valid () || !newph.is_enable ()) {
			PinyinKeyVector keys;
			for (PinyinParsedKeyVector::const_iterator i=pkeys.begin (); i!=pkeys.end (); i++)
				keys.push_back (*i);
			
			/*
			 * If the phrase is in sys phrase library, copy it to user phrase lib.
			 */
			Phrase oldph; 
			
			if (m_sys_phrase_lib && m_sys_phrase_lib->valid ())
				oldph = m_sys_phrase_lib->find (phrase);
			
			if (oldph.valid ()) {
				newph = m_user_phrase_lib->append (oldph, keys);
			} else if (phrase.length () <= m_max_user_phrase_length) {
				
				newph = m_user_phrase_lib->append (phrase, keys);
				
				if (newph.valid () && newph.is_enable ()) {
					// Give an initial frequency to the new phrase.
					uint32 freq = 0;
					if (m_pinyin_table) {
						for (unsigned int i = 0; i < newph.length (); ++ i)
							freq += m_pinyin_table->get_char_frequency (newph [i], pkeys [i]);
						freq /= (1 << (2 * newph.length () - 1));
					}
					
					newph.set_frequency (freq + 1);
				}
			}
		}
	}
	
	if (newph.valid () && newph.is_enable () && refresh) {
		if (newph.length () > 1)
			newph.refresh (SCIM_PHRASE_MAX_FREQ_BITS - m_dynamic_sensitivity);
		else if (m_pinyin_table)
			m_pinyin_table->refresh (newph [0], 31 - m_dynamic_sensitivity, pkeys [0]);
	}
	
	return newph;
}

- (void) clear_selected: (int) clear_pos {
	
    if (!clear_pos) {
        std::vector < std::pair<int, WideString> > ().swap (m_selected_strings);
        std::vector < std::pair<int, Phrase> > ().swap (m_selected_phrases);
    } else {
        size_t i;
        std::vector < std::pair<int, WideString> > tmp_strings;
        std::vector < std::pair<int, Phrase> > tmp_phrases;
		
        for (i = 0; i < m_selected_strings.size (); ++ i) {
            if (m_selected_strings [i].first + m_selected_strings [i].second.length () <= clear_pos)
                tmp_strings.push_back (m_selected_strings [i]);
        }
		
        for (i = 0; i < m_selected_phrases.size (); ++ i) {
            if (m_selected_phrases [i].first + m_selected_phrases [i].second.length () <= clear_pos)
                tmp_phrases .push_back (m_selected_phrases [i]);
        }
		
        m_selected_strings.swap (tmp_strings);
        m_selected_phrases.swap (tmp_phrases);
    }
}




- (void)dealloc {

	[self saveUserLibrary];
	
	delete m_pinyin_global;
	delete m_pinyin_parser;
	

	[super dealloc];
}

@end
